library(plyr)
library(tidyverse)
library(magrittr)
library(broom)
library(lubridate)

d1 <- "https://github.com/thomasjwood/full_fact/raw/main/ff_mt.rds" %>% 
  url %>% 
  gzcon %>% 
  readRDS %>% 
  mutate(
    caseid = caseid %>% as.character,
    gender = gender %>% 
      factor(
        c("Female",
          "Male")
        ),
    out_agree = out_agree %>% 
      mapvalues(
        c("Tend to disagree",
          "Tend to agree"),
        c("Disagree",
          "Agree")
      ) %>% 
      factor(
        c("Strongly disagree",
          "Disagree",
          "Neither agree nor disagree",
          "Agree",
          "Strongly agree")
      ),
    out_true = out_true %>% 
      factor(
        c("True",
          "Probably true",
          "Not sure",
          "Probably false",
          "False") %>% 
          rev
      ),
    agree_num = out_agree %>% 
      as.numeric,
    true_num = out_true %>% 
      as.numeric,
    cond = cond %>% 
      factor(
        c("cond_items",
          "cond_misinfo",
          "cond_corr")
      ),
    educ2 = educ %>% 
      mapvalues(
        
        c("NVQ4 / HNC / HND / Bachelor's degree or similar.",
          "Secondary", 
          "University",
          "High school completed",
          "Complete University", 
          
          "NVQ3/ SCE Higher Grade/ Advanced GNVQ/ GCE A/AS or similar.", 
          "University degree completed",
          "Technikon diploma/degree completed", 
          "NVQ5 or post-graduate diploma.",
          "GNVQ / GSVQ / GCSE/ SCE standard.",
          
          "Complete Secondary",
          "Incomplete University",
          "Artisans certificate, technical/secretarial qualification obtained", 
          "Complete Tertiary education (non-University)",
          "Post graduate degree completed", 
          
          "Incomplete Tertiary education (non-University)",
          "Complete Postgraduate / Master / Doctor's Degree", 
          "NVQ1, NVQ2",
          "Higher non-university",
          "Secondary school (age under 15 years old)", 
          
          "Some high school",
          "Incomplete Postgraduate / Master / Doctor's Degree", 
          "Incomplete Secondary",
          "Primary",
          "Complete Primay",
          
          "Incomplete Primary", 
          "No formal education",
          "Some primary school"),
        
        c("Bachelor's",
          "Secondary", 
          "Bachelor's",
          "Secondary",
          "Bachelor's", 
          
          "Bachelor's", 
          "Bachelor's",
          "Post secondary", 
          "Post secondary",
          "Secondary",
          
          "Secondary",
          "Post secondary",
          "Post secondary", 
          "Post secondary",
          "Master's or higher", 
          
          "Post secondary",
          "Master's or higher", 
          "Secondary",
          "Post secondary",
          "Secondary", 
          
          "Primary",
          "Master's or higher", 
          "Primary",
          "Primary",
          "Primary",
          
          "Primary", 
          "Primary",
          "Primary")
      ) %>% 
      mapvalues(
        c("Bachelor's",
          "Master's or higher",
          "Primary",
          "Secondary"),
        c("Bachelor's or higher",
          "Bachelor's or higher",
          "Secondary or less",
          "Secondary or less")
        ) %>% 
      factor(
        c("Secondary or less",
          "Post secondary",
          "Bachelor's or higher")
        ),
    employ2 = employ %>% 
      mapvalues(
        
        c("Retired",
          "Full-time parent, homemaker",
          "Self employed", 
          "Student/Pupil",
          "Employed full-time",
          "Employed part-time", 
          "Unemployed and not looking for a job/Long-term sick or disabled", 
          "Unemployed but looking for a job"),
        c("Retired",
          "Unemployed",
          "Employed", 
          "Student",
          "Employed",
          "Employed", 
          "Unemployed", 
          "Unemployed")
        ) %>% 
      factor(
        c("Retired",
          "Employed",
          "Unemployed",
          "Student")
        ),
    ideo2 = ideo %>% 
      mapvalues(
        c("Left1", "1", "2", "3", "4",
          "5", "6",
          "7", "8", "9",  "10", "Right10", 
          
          "DK", "Don't know", "Prefer not to say"),
        c("Left", "Moderate", "Right", "DK") %>% 
          rep(
            c(5, 2, 5, 3)
          )
      ) %>% 
      factor(
        c("Left", "Moderate", "Right", "DK")
      ),
    ideonum = ideo %>% 
      mapvalues(
        c("Left1", "1", "2", "3", "4",
          "5", "6",
          "7", "8", "9",  "10", "Right10", 
          
          "DK", "Don't know", "Prefer not to say"),
        c(1, 1:10, 10, NA, NA, NA)
      ) %>% 
      as.character %>% 
      as.numeric,
    age = 2020 %>% 
      subtract(
        birthyr
        )
    ) %>% 
  bind_rows(
    "https://github.com/thomasjwood/full_fact/raw/main/ff_wvs.rds" %>% 
      url %>% 
      gzcon %>% 
      readRDS %>% 
      select(S020, S003, X002, X025, X028, X001, E033, S017) %>% 
      rename(
        year = S020,
        country = S003,
        birthyear = X002,
        educ2 = X025,
        employment = X028,
        gender = X001,
        ideo = E033,
        weight = S017
      ) %>% 
      mutate(
        age = year %>% 
          as.character %>% 
          as.numeric %>% 
          subtract(
            birthyear %>%
              as.character %>%
              as.numeric
            ),
        ideo_num = ideo %>% 
          mapvalues(
            c("Missing; Unknown", "Not asked", "No answer", "DonÂ´t know", 
              "Left", "2", "3", "4", "5", "6", "7", "8", "9", "Right"),
            c(NA,
              1:10) %>% 
              rep(c(4, 1 %>% rep(10)))
            ) %>% 
          as.character %>% 
          as.numeric,
        ideo2 = ideo %>% 
          mapvalues(
            c("Missing; Unknown", "Not asked", "No answer", "DonÂ´t know", 
              "Left", "2", "3", "4", "5", "6", "7", "8", "9", "Right"),
            c("DK",
              "Left", "Moderate", "Right") %>% 
              rep(c(4, 4, 2, 4))
          ) %>% 
          factor(
            c("DK",
              "Left", "Moderate", "Right")
            ),
        employ2 = employment %>%
        mapvalues(
          c("Missing; Unknown", "No answer", "Full time", "Part time",
            "Self employed", "Retired", "Housewife", "Students", "Unemployed",
            "Other", "9"),
          c(NA, NA, "Employed", "Employed",
            "Employed", "Retired", "Employed", "Student", "Unemployed",
            NA, NA)
        ) %>%
        factor(
          c("Retired",
            "Employed",
            "Unemployed",
            "Student")
        ),
        educ2 = educ2 %>% 
          mapvalues(
            c("Missing; Unknown",
              "Not asked in survey", 
              "Not applicable; No formal education", 
              "No answer",
              "DonÂ´t know",
              
              "Inadequately completed elementary education", 
              "Completed (compulsory) elementary education",
              "Incomplete secondary school: technical/vocational type/(Comp", 
              "Complete secondary school: technical/vocational type/Seconda", 
              "Incomplete secondary: university-preparatory type/Secondary,", 
              
              "Complete secondary: university-preparatory type/Full seconda", 
              "Some university without degree/Higher education - lower-leve", 
              "University with degree/Higher education - upper-level tertia"),
            c(NA,
              NA, 
              "Secondary or less",
              NA,
              NA,
              
              "Secondary or less",
              "Secondary or less",
              "Secondary or less",
              "Secondary or less",
              "Secondary of less",
              
              "Post secondary",
              "Post secondary",
              "Bachelor's or higher"
              )
            ) %>%
          factor(
            c("Secondary or less",
              "Post secondary",
              "Bachelor's or higher")
            )
        ) %>% 
      select(
        -year, -birthyear
      ) %>% 
      mutate(
        wave = "wvs",
        caseid = "wvs_" %>% 
          str_c(
            "https://github.com/thomasjwood/full_fact/raw/main/ff_wvs.rds" %>%
              url %>% 
              gzcon %>% 
              readRDS %>% 
              nrow %>% 
              seq
            )
        )
    ) %>% 
  mutate(
    country = country %>% 
      mapvalues(
        "nigeria",
        "Nigeria"
        ),
    weight = weight %>% 
      as.character %>% 
      as.numeric
    )



t1 <- d1 %>% 
  select(
    weight, caseid:country, age, gender, educ2:ideo2
  ) %>%
  mutate(
    weight = weight %>% 
      is.na %>% 
      ifelse(1, weight)
  ) %>% 
  group_by(
    caseid, wave
  ) %>% 
  slice(1) %>% 
  ungroup %>% 
  left_join(
    d1 %>% 
      filter(
        wave == "wave 1"
      ) %>% 
      select(-wave) %>% 
      group_by(
        caseid
      ) %>% 
      summarize(
        corr = cond %>% 
          str_count("_corr") %>% sum,
        country = country %>% first
      ) %>% 
      mutate(
        cnum = corr %>% 
          cut_number(
            n = 2,
            labels = c("0-2", ">2")
            )
        ) %>% 
      select(-corr)
    ) %>%
  mutate(
   age = age  %>% 
      cut(
        breaks = c(-Inf, 26, 40, 60, Inf),
        labels = c(
          "18-26", "27-40", "41-60", "61+"
        )
      )
   ) 


t2 <- t1 %>% 
  gather(
    cat, val, c(gender:ideo2, age), na.rm = T
  ) %>% 
  mutate(
    val = val %>% 
      factor(
        t1 %>% 
          select(
            c(gender:ideo2, age)
            ) %>% 
          map(
            levels
          ) %>% 
          unlist %>% 
          as.character
        )
    ) %>% 
  group_by(
    country, wave, cnum, cat, val
  ) %>%
  tally(weight) %>% 
  mutate(
    perc = n %>% 
      divide_by(n %>% sum) %>% 
      multiply_by(100) %>% 
      round
  ) %>% 
  unite(
    cwc, country:cnum
  ) %>% 
  mutate(
    cwc = cwc %>% 
      factor(
        c("Argentina_wvs_NA", 
          "Argentina_wave 1_0-2",
          "Argentina_wave 1_>2", 
          "Argentina_wave 2_0-2",
          "Argentina_wave 2_>2",
          
          "South Africa_wvs_NA", 
          "South Africa_wave 1_0-2",
          "South Africa_wave 1_>2", 
          "South Africa_wave 2_0-2",
          "South Africa_wave 2_>2",
          
          "United Kingdom_wvs_NA",
          "United Kingdom_wave 1_0-2", 
          "United Kingdom_wave 1_>2",
          "United Kingdom_wave 2_0-2",
          "United Kingdom_wave 2_>2", 
          
          "Nigeria_wvs_NA",
          "Nigeria_wave 1_0-2", 
          "Nigeria_wave 1_>2")
      )
  ) 


t_t <- t1 %>% 
  select(
    weight:wave, country, cnum
    ) %>%
  distinct %>%
  group_by(
    country, wave, cnum
  ) %>%
  tally(weight) %>% 
  unite(
    cwc, country:cnum
  ) %>% 
  mutate(
    cwc = cwc %>% 
      factor(
        c("Argentina_wvs_NA", 
          "Argentina_wave 1_0-2",
          "Argentina_wave 1_>2", 
          "Argentina_wave 2_0-2",
          "Argentina_wave 2_>2",
          
          "South Africa_wvs_NA", 
          "South Africa_wave 1_0-2",
          "South Africa_wave 1_>2", 
          "South Africa_wave 2_0-2",
          "South Africa_wave 2_>2",
          
          "United Kingdom_wvs_NA",
          "United Kingdom_wave 1_0-2", 
          "United Kingdom_wave 1_>2",
          "United Kingdom_wave 2_0-2",
          "United Kingdom_wave 2_>2", 
          
          "Nigeria_wvs_NA",
          "Nigeria_wave 1_0-2", 
          "Nigeria_wave 1_>2")
      )
  ) %>% 
  na.omit


t2 %>% 
  select(-n) %>% 
  spread(cwc, perc) %>% 
  as.data.frame

